# to statistics the domains

require "dbm"
require "00_common"

PAYLOAD_SPLITER = "\\r\\n"

module DomainStat	
	def self.extract_request(payload_string)
		fields = payload_string.split PAYLOAD_SPLITER
		return fields[0]
	end

	def self.extract_host(payload_string)
		fields = payload_string.split PAYLOAD_SPLITER
		i = 0

		rzt = "NULL"
		while i < fields.length and fields[i][0,4] != "Host"
			i = i+1
		end

		if i < fields.length
			rzt = fields[i]
		end

		rzt
	end

	def self.extract_referer(payload_string)
		fields = payload_string.split PAYLOAD_SPLITER
		i = 0

		rzt = "NULL"
		while i < fields.length and fields[i][0,7] != "Referer"
			i = i+1
		end

		if i < fields.length
			rzt = fields[i]
		end

		rzt
	end

	def self.get_all_payload_files
		Common.get_all_files "./payloads"
	end

	def self.handle_each_file(file_name)
		puts "handle file:#{file_name}..."
		file = File.open "./payloads/" + file_name

		# output files
		map_protocol = Hash.new
		map_domain = Hash.new
		map_l4 = Hash.new

		file_http_request = File.open "http_request_" + file_name, 'w'

		counter_total = 0
		counter_empty = 0

		begin # handle each line
			while line = file.readline and line.length != 0
				if line[-1,1] == "|"
					line = line + "NULL"
				end

				counter_total = counter_total + 1
				# puts counter_total
	
	      # local_time = fields[LOCAL_TIME_STAMP].length == 0 ? "NULL" : fields[LOCAL_TIME_STAMP]
	      # server_ip = fields[SERVER_IP].length == 0 ? "NULL" : fields[SERVER_IP]
	      # l4_proto = fields[L4_PROTO].length == 0 ? "NULL" : fields[L4_PROTO]
	      # protocol = fields[PROTOCOL].length == 0 ? "NULL" : fields[PROTOCOL]
	      # cpayload = fields[CLIENT_PAYLOAD].length == 0 ? "NULL" : fields[CLIENT_PAYLOAD]
	      # spayload = fields[SERVER_PAYLOAD].length == 0 ? "NULL" : fields[SERVER_PAYLOAD]
	      fields = line.split "|"

	      local_time = fields[0]
	      server_ip = fields[1]
	      l4_proto = fields[2]
	      protocol = fields[3]
	      cpayload = fields[4]
	      spayload = fields[5]

      	# puts "===> protocol:" + protocol
      	# stat for protocol
      	if map_protocol[protocol]
      	  map_protocol[protocol] = map_protocol[protocol] + 1
        else
        	map_protocol[protocol] = 1
      	end

      	# puts "===> l4:" + l4_proto
      	# stat for l4
      	if map_l4[l4_proto]
      	  map_l4[l4_proto] = map_l4[l4_proto] + 1
        else
        	map_l4[l4_proto] = 1
      	end

	      # puts "http"
      	# extract http payload
      	if protocol == "HTTP"
      		# get the domain

      		domain = "NULL"
					request = "NULL"
					referer = "NULL"
      		if cpayload and cpayload.length != 0 and cpayload != "NULL"
      			#puts "<<<<<   extract >>>>>>>"
      			# puts cpayload

	      		host_line = DomainStat.extract_host(cpayload)
	      		# puts host_line
	      		if host_line and host_line.length > 6
		      		domain = host_line[6, (host_line.length - 6)]
	      		end

      			#puts "<<<<<   extract domain >>>>>>>"
	      		#puts domain
	      		request = DomainStat.extract_request(cpayload)

      			#puts "<<<<<   extract request >>>>>>>"
	      		#puts request

	      		referer = DomainStat.extract_referer(cpayload)
      			#puts "<<<<<   extract referer >>>>>>>"
	      		#puts referer
					end


      		#
      		# output the http information
      		#
      		file_http_request.puts local_time + "|" + domain + "|" + request # + "|" + DomainStat.extract_date(spayload)

      		#
      		# stat for domain
      		#
    			if map_domain[domain]
    			  map_domain[domain] = map_domain[domain] + 1
    		  else
    		  	map_domain[domain] = 1
    			end
	      else
	      	counter_empty = counter_empty + 1
	      end
			end # of while
		rescue Exception => e
			if e.to_s == "end of file reached"
				puts "end of file"
        file.close
			else
				puts "ERROR:"
			  puts e.backtrace
			end
		end # of handle each file

		#
		# After the file is done
		#
		file_http_request.close
		Common.move_file("http_request_" + file_name, "./http_request")

		Common.output_hash_to_file(map_protocol, "protocol_" + file_name)
		Common.move_file("protocol_" + file_name, "./stat_protol")

		Common.output_hash_to_file(map_domain, "domain_" + file_name)
		Common.move_file("domain_" + file_name, "./stat_domain")

		Common.output_hash_to_file(map_l4, "l4_" + file_name)
		Common.move_file("l4_" + file_name, "./stat_l4")

		puts "#{counter_total} in total. #{counter_empty} for empty"
	end
end

begin
	all_files = DomainStat.get_all_payload_files
	# all_files = ["payloads_RawStreams-ls-0-1293649086-2724.bcp"]

	all_files.each do |file|
		# puts file
		DomainStat.handle_each_file file
	end

rescue Exception => e
	puts "ERROR:" + e.to_s   
end